{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = 'mesolitica-tpu.json'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import tensorflow_datasets as tfds\n",
    "from t5.data import preprocessors as prep\n",
    "import functools\n",
    "import t5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gin\n",
    "\n",
    "gin.parse_config_file('gs://mesolitica-tpu-general/t5-data/pretrained_models_base_operative_config.gin')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "vocab = 'gs://mesolitica-tpu-general/t5-data-v2/sp10m.cased.ms-en.model'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def dumping_dataset(split, shuffle_files = False):\n",
    "    del shuffle_files\n",
    "    files = [\n",
    "        'gs://mesolitica-tpu-general/t5-data-v2/dumping-news.txt.tsv',\n",
    "        'gs://mesolitica-tpu-general/t5-data-v2/dumping-parliament.txt.tsv',\n",
    "        'gs://mesolitica-tpu-general/t5-data-v2/filtered-dumping-academia.txt.tsv',\n",
    "        'gs://mesolitica-tpu-general/t5-data-v2/filtered-dumping-wiki.txt.tsv'\n",
    "    ]\n",
    "    files.extend(tf.io.gfile.glob('gs://mesolitica-tpu-general/t5-data-v2/00.jsonl-*.translated.txt.tsv'))\n",
    "    ds = tf.data.TextLineDataset(files)\n",
    "\n",
    "    ds = ds.map(\n",
    "        functools.partial(\n",
    "            tf.io.decode_csv,\n",
    "            record_defaults = ['', ''],\n",
    "            field_delim = '\\t',\n",
    "            use_quote_delim = False,\n",
    "        ),\n",
    "        num_parallel_calls = tf.data.experimental.AUTOTUNE,\n",
    "    )\n",
    "    ds = ds.map(lambda *ex: dict(zip(['title', 'text'], ex)))\n",
    "    return ds\n",
    "\n",
    "\n",
    "t5.data.TaskRegistry.remove('dumping_dataset')\n",
    "t5.data.TaskRegistry.add(\n",
    "    'dumping_dataset',\n",
    "    dataset_fn = dumping_dataset,\n",
    "    splits = ['train'],\n",
    "    text_preprocessor = functools.partial(\n",
    "        t5.data.preprocessors.rekey,\n",
    "        key_map = {'inputs': None, 'targets': 'text'},\n",
    "    ),\n",
    "    token_preprocessor = t5.data.preprocessors.unsupervised,\n",
    "    sentencepiece_model_path = vocab,\n",
    "    metric_fns = [],\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/ops/array_ops.py:1475: where (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n",
      "INFO:tensorflow:tokens_length=1137 inputs_length=1024 targets_length=229 noise_density=0.15 mean_noise_span_length=3.0 \n",
      "{'inputs': array([   14,   494,   270,     7,   883,    17,   964,    24,   272,\n",
      "       10496,     3,  4500,    83,  1105,    14,   158,  2255,    41,\n",
      "         307,    14,  5400, 32099, 10735,  5763,    34,   157, 10237,\n",
      "        8159,    17,  9804,     3,   418,  1232,  7478,   208, 24093,\n",
      "          24, 11009,   983,  9447,  3914,  2607, 22725,  7313, 12451,\n",
      "        1327,  2821,   125, 22725,  6166,  1056, 32098,     3,  7884,\n",
      "        3540, 32097,  7329,    42,  3099,  3105,    62,  7925,   179,\n",
      "           3,   276,  1773,    73,    25,  4282, 11762, 32096,  1650,\n",
      "        1434,  1451,     3,  1865,   522,    37,    14,   179,     3,\n",
      "        1727, 32095,  4282, 11762,  6989,    14,    87, 11479,  4282,\n",
      "       32094,  6989,    14,    87,   195,  1974,   200,   793,  2427,\n",
      "       14424, 11762,  6989,     3,  1481,    91, 32093,  7503,   987,\n",
      "       16054,    13, 14157, 13213,    47,   116,    17,   742,  4282,\n",
      "       11762,  6989,     3,  2004,    37,    14,    46,    93, 12707,\n",
      "          24,  8059,    22,   296,  3735,  1650,    55,     7,  2310,\n",
      "          25,  5413,   987, 16054,    13, 14157, 13213,    47,    13,\n",
      "        1062,  7340,    55,  1275,    16, 19315,   348,   440, 27311,\n",
      "         103,  5509, 22888,     3,  9077,  6185, 10888,   724,    16,\n",
      "         668,    14, 17045,   450, 27149,  6185,  2540,    24,   398,\n",
      "       21903,     3,   231,    13,     7,  9077,  4773,    25,    30,\n",
      "         315,    13,    94, 32092,   247,   665, 13297, 19413, 27149,\n",
      "       16754,    87, 17045,   450, 27149,    42,   602,  4785,    14,\n",
      "         312,    17, 32091, 29506,     3, 32090,  1181,  5062,  6070,\n",
      "       12822,  1524,  5541,  2580,  3141,    13,     4,   386,   829,\n",
      "        3655,     5,    14,    13, 10434,  3075, 22142,  3059,   294,\n",
      "         498,  1278,   499,     3,  1892,    16,  3234,    14,  1263,\n",
      "          34,    30,    97, 20363,  2367,    17,   851,    24,   398,\n",
      "         987,  6205, 10450,  4241, 27149,     3,    13,     6,  2986,\n",
      "       10215,  2804,   377,    13,  2371, 32089,    13,    94,    16,\n",
      "         668,     3, 12342,   115,   344, 32088,  3888,  2576,  1545,\n",
      "          36, 12690,    17,  3888,   578,    52, 11051,    38,    14,\n",
      "           6,    98,  3075, 22142,    33,  5961,    83, 32087, 13297,\n",
      "       11815, 32086, 27149,    14,  1075,    13,     4,  2732,   140,\n",
      "       27792, 32085,     3,  8683,   490,   573,    17,  2870,    60,\n",
      "        1524,  5541,  2580,  3141,    13,  3963, 32084,    17, 32083,\n",
      "         987,  6205, 10450,    13, 11276,  8180, 27149,    14,   758,\n",
      "          93,    28,   845,    24,   138,   398,   280,     3,  2830,\n",
      "          93,    31,  2386,   258,   159, 17045,   450, 27149,    87,\n",
      "         434,   712,  1165, 20945,   602,  4785,    17, 16593,   777,\n",
      "         890,  3386,   918,    14,  1553,    17,  3233,  7857,  1470,\n",
      "          60,   845,    52, 32082,  1288,    33,   322,  1747,    14,\n",
      "       14592,   152,  7324,  8690,   476, 30749,    17,  1637,  1320,\n",
      "          36,  1627,  3546,  3025,  7737,     3,    13,     6,  4926,\n",
      "           7,   704,    33,    13,    94,    16,   668,    14,    37,\n",
      "        4074,    13,     4, 10995,     5, 14424,  4488,  3862,    13,\n",
      "       32081,  1752,   125,  3242,    47,  1320, 32080,   398,    14,\n",
      "           6, 29451, 32079,     3,   857, 32078,    14,   764,   321,\n",
      "       10356,    24,   362,    83,    17,    97,  1489,  1320,    42,\n",
      "         312,    93,   391, 32077,    13,  3491,   882,    28,   311,\n",
      "         671,   430,   519, 12974,    15,   222,  4096, 30094, 17936,\n",
      "        1627,   429,   519,  6626,  1290,  7035,     3,  2830,    93,\n",
      "          31, 17045,   450, 27149,   208, 12295,  1352,   104,  2388,\n",
      "        6626, 22943,  1033,    14,   290,    17,  2206,  8936,  9773,\n",
      "       13811,   924,  6889, 32076,    34,    14,    13,  8999,  7077,\n",
      "          13,     4, 32075, 22723,    47,  1226,   188,   590,  3998,\n",
      "         354,   398,     5, 13534,  2367,   245,   331,  1553,    14,\n",
      "       32074,  4011, 12862,     3,  9077,   800,    39,     1]), 'targets': array([32099,    13, 32098,  1637,    36, 22888,  1919, 18657, 32097,\n",
      "        2256, 10865,    16, 32096,  6989,   156, 12006,  3458,  3735,\n",
      "       32095,   343,    62, 32094, 11762, 32093,   718, 30913, 32092,\n",
      "          16,   668, 32091,    62, 21053,    55,  7324,   138, 32090,\n",
      "       27273, 10005,    22, 14751, 32089, 17036,   150, 32088, 17000,\n",
      "         103,   845,  2401, 32087,   344,    16, 32086,  4241, 32085,\n",
      "       25279,     5, 32084,    14,   742,    97, 12643,   780,   398,\n",
      "         187, 32083,   288,    24,   312,    17, 32082,   497,   245,\n",
      "        1161,  8214, 32081,     4, 32080,     5,    55, 32079,    38,\n",
      "       32078,  2695,    37, 32077,  4332, 32076,  1553, 32075,  2598,\n",
      "       32074,   240,   116,    31,   179,     3,  9077, 32073,  1525,\n",
      "       32072,    13, 10210,  7092,   223,     3, 32071, 14552, 19542,\n",
      "        1080, 32070,  1518,  1080,    28,   410, 32069,    28,   410,\n",
      "          22,  6935, 32068,     3, 32067,    22,    50,   814, 32066,\n",
      "         231,    14, 32065,     5,    13,  1987, 32064,  9756,    14,\n",
      "        3397, 32063,   741, 11301, 32062,    46,   440, 32061, 16941,\n",
      "          17,    24, 11038,  8139, 15257,  2487,  2751,     3, 16481,\n",
      "       26594, 23611,    16, 32060,   530, 32059,  1821, 32058,    13,\n",
      "           4,  1910, 32057,     4,  1817,     5,    14,   249, 32056,\n",
      "          13,     6,  5628,  4429,   464,   445,  8938, 32055,    36,\n",
      "        4276, 32054,    14, 29023,    14,  4771,   342,   429, 32053,\n",
      "          22, 32052, 13912,    14, 32051,  1836,    22, 11171, 32050,\n",
      "          16,    47,     3, 32049,     3, 32048,   227,   527,  8978,\n",
      "         103,    33,  3129, 32047,  2112, 32046, 18340,    13,     4,\n",
      "        1910, 32045,     5, 32044,   445, 32043,  6109,  3353,    83,\n",
      "       10762,    25,  1771,     1])}\n",
      "{'inputs': array([   81,   284,   349,  1229,  1338, 21562,  7803,  5631,  7035,\n",
      "         987, 24141,    91,   348, 13660, 12366,    95, 28681,     6,\n",
      "           3, 17191,  1882,   303,  7147,   682,  7833,    14,  1844,\n",
      "        7508,    17, 12166,  5572,  1886,  7833,    38,  1374,    22,\n",
      "        9939, 32099, 27069,    14,    33,  3129,  1553,   212,   797,\n",
      "        1051,     3, 32098,    93,   505,  3758,    13,  1476,  2601,\n",
      "          17,   176,   187,    25, 32097,  4473,   169,    37,    14,\n",
      "          13,  8562,  3964,    91,    36,  9939,  7833,    14,   105,\n",
      "        7803,    14, 12095,   372,  1773,    14,    13, 32096,    41,\n",
      "       32095,    22, 32094,   190,  7035,    55,    38,  2460,    65,\n",
      "         461,    14,  1305, 19271,  7803,    22, 12095,   372,  1773,\n",
      "          52,  5260, 32093, 12366,    22,  2243,  1611,  1294,    14,\n",
      "           6, 32092,     3,  2004,    37,    14,  1972,  1811,  1070,\n",
      "        1882, 13816,   354,   464,  2231,  3879,    47,  7833,    17,\n",
      "          45,    24,  4603,  7147,   682,     3,   414,   705,    47,\n",
      "         177,  4429,    17,   342,  1476,  2601,   273,  2231,  3879,\n",
      "        3025,    30,  1051,    36,  1404, 32091,  3888,    95,    50,\n",
      "       15188,  1670,   797,   187, 32090, 10666, 32089,  8236,  1101,\n",
      "          17,    65,  7567, 30321,   123, 10666,   294,  2371,  7833,\n",
      "          14,   105,  1101, 11240,    87, 19754,  2931,     6,     3,\n",
      "       23951,  3346,    93,  8614,  1882,   303,  7147,   682, 32088,\n",
      "        6482,  4074,  3509,   987,  4603,   452, 20307,    13,  3775,\n",
      "        6437, 26774,  6324,     3,  2386,   303,  7147,   682,  7833,\n",
      "          17,   391,   987,  4603,   452, 20307,    13,  3775,  6437,\n",
      "          90,   864,    13,  5733,  1670,  4808,   105,  6376,  7800,\n",
      "          47,    14,    61,    94,    14,    13,   188,  2775,  9259,\n",
      "          22, 17129,    14,     6,    13, 16226,  5460,    38,     3,\n",
      "        1181,  5524,    76,  5091,  1629,  3146,  3514,    83,  7916,\n",
      "         602, 32087,  2225,    14,   615,   398,    13,     7,     7,\n",
      "         506,  8870,    13,     4,     5,   643,  3590,  8837, 13187,\n",
      "          91,   101,    45, 18276,  2044,    55, 27807,    47,    13,\n",
      "           4,     5,    36,   245,  2825,   226,  7916, 32086,   138,\n",
      "         398,    13,     4,     5,  3891,   717,  1764,   621,    18,\n",
      "        1253,    20, 22126,   261,    13,     4,     5,     3,   643,\n",
      "        3590,  8837,   101,  2825,   226,  7916, 32085,  9188,  2665,\n",
      "         520,    14,  5816,    30,   779,   578,     3,  9827, 16605,\n",
      "          73,     7,   704,    37, 32084, 10555,    17,  7593,     6,\n",
      "           3,  2089,    45,   422,  8625,   103,  1126, 32083,    14,\n",
      "          28, 10314,    13,  3777,   125,  1476,   173,  9890,   845,\n",
      "           3,   563,   169,   764,    14,  5816,    93,   876,  5638,\n",
      "          22,    84, 32082, 16371,    14,     6,    98,   643,  3590,\n",
      "        8837,    36,  4869, 23666,    24,  9661,    16, 19768,   386,\n",
      "         829,  5524,    17, 16743,   615, 18461,     3,   231,    14,\n",
      "         989,    13,     4,   343, 32081,  2825,   226, 32080,  7539,\n",
      "        1133,  3668, 10266,  3546,  2206,   294,  2545,    13,     4,\n",
      "        4157,  9417,     5,     3,   160, 32079,     7, 21521,    47,\n",
      "       13297, 11568,  9756,  7916,   602,  9220,   212,     3, 17191,\n",
      "          34,    14, 32078,    14,   764,   274,  6149,  1126,    17,\n",
      "          45, 32077,  3590,  8837,    13, 21069,  1627,  3546,    47,\n",
      "       25738,  7916,   602,  9220,  9188,  2665,   288,    24,   292,\n",
      "       22282,  1181,  1232,  8813,  1775, 11300,  7716,  5437,    79,\n",
      "       13552,     6,     3,    13, 23703, 11568,  9756,    46,    37,\n",
      "         742, 15492,   764, 32076, 15769,   156, 22997,    60,  7324,\n",
      "          23,  9444,   682, 13297,    24,   292,    13, 18440,  1181,\n",
      "        1232,  8813,  1775, 11300,     3,    13, 32075,   108,  1553,\n",
      "          17, 11210,    14,   187, 23632,  3023, 14592,   152,  4242,\n",
      "          38,    14,     6, 32074, 24133,   742,    97,     1]), 'targets': array([32099,    38, 32098,  9290,    63,  7833, 32097, 20857,     3,\n",
      "       32096,  1594, 32095,  8140,    14, 32094, 12095,   447,    14,\n",
      "       32093,   520,    25,  2711, 32092,    13, 14015,    38, 32091,\n",
      "        1110,    93, 32090,     3,  5144,  3420,   474, 32089,   294,\n",
      "        2371,  7833,    14, 32088,  7833, 32087,  9220, 20827,  2665,\n",
      "       32086,    73,  2367, 32085,    73, 32084,    62,    13, 17229,\n",
      "        2669, 32083,   736, 32082,  3146, 32081,   140,   215,     5,\n",
      "           3,   643,  3590,  8837,   101, 32080,   445,  9188,  2665,\n",
      "          37,   157,   605,    47,    42, 32079,  3234,   386,   829,\n",
      "        5524,  2138,   322,   226,   605,    47, 32078,    98,    61,\n",
      "       32077,   578,     3,   643, 32076,    22,   274, 32075, 16200,\n",
      "          97, 32074, 27320,    61,     3, 32073,    94,    41,    14,\n",
      "         386, 32072,  1744,   103, 32071,     3, 32070, 20247, 21039,\n",
      "        1697,    13,     4, 32069,  6034,   506, 13174,    22,  3609,\n",
      "       32068,  2985,  3365, 11278,   411, 32067,   186, 32066,     3,\n",
      "          13, 32065,    37,  6676,  6555, 32064,    30,   631,   291,\n",
      "         177,     7, 32063,   453,  6894, 32062,    61, 32061,  7737,\n",
      "           6, 32060,   105, 32059,   987,   453,  6894,    71, 19394,\n",
      "         280,     6,     3,  2569,   259, 10165,    22,    67,  2319,\n",
      "        1287, 32058,   204,   186,    14, 32057, 11083, 32056,    14,\n",
      "       30552,   190, 32055,  1708,  2946, 32054,   429, 32053,  1355,\n",
      "         881, 32052, 12869,  3749, 32051, 18240,     3, 32050,  3446,\n",
      "       32049, 24678, 32048,  4376,    13,  1125, 10737,    47,  3740,\n",
      "          13, 32047,     5,     3, 32046,  1215, 14908,    13,  9975,\n",
      "          22,   391,  3888, 32045, 24819, 32044,    13,  9975,    14,\n",
      "       32043,   489,    91,     1])}\n",
      "{'inputs': array([  790,  1111,   107,  1183,    19,   383,   523,     3, 29080,\n",
      "       32099,   196, 32098,   134,    64,   656,    58,  1111, 18450,\n",
      "         946, 16058,    14,    40,   898,  2254,    19, 22666,  7846,\n",
      "          20,  3178, 32097,  1498,    13,     6,   117,  6770,    41,\n",
      "          64,  3536,    19,    15, 31293,  3933,    18,    15,  4241,\n",
      "          14,     6,   148,    49,     3,   104,    21,   873,    14,\n",
      "       10411,  1260, 32096,  2497,    13,     4,   133, 11799,     5,\n",
      "          49,    88,    13,     6,  7075,  4103,    15,  2326,  1150,\n",
      "         297,   508,     6, 32095,   483, 32094,    27,    13,     6,\n",
      "         619,   236,   932,  3425, 20627,  1498,    18,    15, 32093,\n",
      "          50, 12315,    14,    96,    13,     6,  7666,  9524,    16,\n",
      "          27,    15,  2797,    20,   316,     7, 10018,    18,   127,\n",
      "        7054,    20,    15,  1036,   383,  1660,  8282, 32092,   175,\n",
      "        5065,  1282,    88,   134,    50, 10336,   131,    19, 20648,\n",
      "          14,    20, 32091, 15879,    23,    15,  1934,  1904,   966,\n",
      "          15,  9225,    16,    13,     6, 28912,  8008,    50,  1168,\n",
      "         775,    23,    15,   191,    14,    96,    23,   300,   713,\n",
      "          69, 32090,  1587,  3229,     6,     3,  1401, 13320,    63,\n",
      "          13, 19095,    16,  2485,  1297,    14,  3112,  2023, 32089,\n",
      "       13019,   144,    39,    15,   403,   146,  1327,    91, 13715,\n",
      "        1195,    63, 32088,    15,  6067,  1044,    27,   207,  3425,\n",
      "       20627,  3081,    26,    44,   812,    94,  4066,   766,    44,\n",
      "          15,  5625,  1597,  2979,  2182,    16,    14,   124, 19400,\n",
      "          16,   853,     7,   675,  4678, 23999,    16,    58,  1192,\n",
      "          14,  3847,    14,  6745,   925,    20,    15,   819,    16,\n",
      "          23,  1731,    18, 32087,   263,    39,    21,    13,  2414,\n",
      "       13238,    63, 24600,    29,    15,   175,     7,  1442, 32086,\n",
      "          19,  2002,    92, 14277,    32,  6255,   181,  1225,  1162,\n",
      "         277, 32085,    15,  1271,    29,   119,    13, 24392,    23,\n",
      "       19064,    32,    15,   174,   403,    18,    15,  4391,    16,\n",
      "       32084,  7502,  1998,  1498,    23, 11556,    14,   786,  1142,\n",
      "          14,    32,  1077,     3,    35,    13,  1022, 32083,    15,\n",
      "        1401, 13019,    39,    27,    88,  3095,    92,  2288,    19,\n",
      "          13,     6, 23014,    16,   654, 32082,     6,    40,    21,\n",
      "       18402,  1679,    51,  1679, 15062,    19,    81,    96,   378,\n",
      "        9025,    16, 10025,  4239,   181,    92,  1162,   146,  1327,\n",
      "          91, 17157,    15,  7347, 15062, 13333,     3,   465,    92,\n",
      "        1657,    14,  1327,    91,    20,  3535,  7322,    19,    15,\n",
      "          13, 10445,    41,    40,    21,  7570,  4563,    20,  6570,\n",
      "          40,    80,  6976,  6447,   687, 15062, 12620,    14,    21,\n",
      "        1231,     7, 14289,  1158, 22744,    20, 16411,    13,  4868,\n",
      "        2315, 11445,     3,  1327,    91, 13715,  1195,    63,    13,\n",
      "           4,   586,     5, 32081,   389,  1360,    23,  1485,   326,\n",
      "          15, 15015,  5625,  1597,  2979,  2182,    16, 24712, 23620,\n",
      "        7714,    16,    48,  1275,   188,  7117, 13857, 14192,   188,\n",
      "          23, 11556,    14,  1978,  6562,  3229,     3,  1327,    91,\n",
      "       13715,  1195,    63,    13, 32080,    20,  3535,   389,  1360,\n",
      "          23,  1485,   326,    15, 32079,  5625,  1597,  2979, 32078,\n",
      "          16,    48, 32077,   188,  7117, 13857, 14192,   188,    23,\n",
      "       11556,    14,  1978,  6562, 32076,  5369,    14,    29,   127,\n",
      "          15,  6320,    20, 28580,    14,    15,  4525,    66,   114,\n",
      "       32075,   483,   195, 32074,  1976,    20,  2234, 29241,  5324,\n",
      "           7,  3598,  8150,    16,    20,   934, 19161,   814,   181,\n",
      "          13,  4791,  3997,  2474,    44,    15,  4055,  4391, 28897,\n",
      "          72,    19,   174,   661,    40,    21, 15062, 32073,    13,\n",
      "       17473,    20,    21,  1679,     7,  1732,     7,  3103, 15062,\n",
      "          19,    81,    13,   258, 32072,   814,  9124,     1]), 'targets': array([32099,    49, 32098, 24270, 32097,     3,   414,   299,    14,\n",
      "          15, 32096,  2201,    20,    15,  1193, 23620, 32095,    51,\n",
      "        9293,  1276,     3, 32094,    49,    88, 11440, 32093,  1015,\n",
      "           6,  1021, 32092,  3734,     6,     3,    35, 32091,    13,\n",
      "       32090,   277, 32089,    19,  2593,   140,   639,   153, 32088,\n",
      "          20,  3535,   389,  1360,  7053, 32087,    21,  2511,     7,\n",
      "        1774,  4582,     3, 32086,    20, 13320,    63,   340, 18952,\n",
      "          14,    86, 13429, 32085,  4902, 32084,  1297,   338,    48,\n",
      "          15,   369,     7,  6664, 32083,   562,  8946,    29, 32082,\n",
      "       21696,    81, 32081,    20,  3535, 32080,     4,   586,     5,\n",
      "       32079, 15015, 32078,  2182,    16, 24712, 23620,  7714, 32077,\n",
      "        1275, 32076,  3229,     3, 32075,  1329,     3, 32074, 20802,\n",
      "       11198,     3,  1845, 32073, 12620,    14,    21,  7347, 15062,\n",
      "       32072,     7, 32071, 15845, 10107,     3,  4410, 32070, 10146,\n",
      "       32069,    12, 32068,    13, 32067,  3642,    15,   403, 32066,\n",
      "          18,  5137, 32065,     4,   702, 32064, 24712, 23620, 32063,\n",
      "          23, 32062,  6562,  3229,     3,  2593, 32061,     5,    20,\n",
      "       32060,   174,   403, 32059,  2979,  2182, 32058, 23620,  7714,\n",
      "          16,    48, 32057,    19, 32056,    13, 32055,     3,   421,\n",
      "        1373,   243,  6186,    26, 32054,   165,    14,     6, 32053,\n",
      "          48,    15,   658,    18,   239,   191, 32052,  2069, 32051,\n",
      "         587, 32050,  8686,   607,    14,    21, 15062, 32049,    20,\n",
      "          21, 15062,  1599,  2539, 17549,  2240,   120, 22929, 32048,\n",
      "          19,    13, 32047,  5717, 32046,   435,  4471,   680, 32045,\n",
      "         107,  3843, 32044,   587,    12,    16,  1498,   326,    15,\n",
      "       15015, 32043,    14,     1])}\n",
      "{'inputs': array([ 5342,   155,    17,   200,    14,    67, 32099,    36,  6748,\n",
      "         906,    14,    52,   176, 32098,    14,     6,   194,   123,\n",
      "        2822,  9875,     3,   857,    37,    14,    28,   342, 11974,\n",
      "       10274,    55,  5779,  4886,   494,  9243, 32097,    14, 27107,\n",
      "        1666,  1557, 10963,   302,   226,  2293,  1384,     3,  3479,\n",
      "        6196,    55,  8679,  2750,    47,    14,  3535, 20074, 16303,\n",
      "        9271,  1935, 10687,    13,     4, 13475,     5,    22, 27107,\n",
      "       32096,   385,  2271,   847,  2293, 15435,  1611,    85,  4226,\n",
      "           3,    13,     7,  3922, 11350, 16010,    13,     6,  1892,\n",
      "       32095, 11784,   845,    14,  7935, 20123,  2427,   315, 18622,\n",
      "          87,  4894,    14, 32094,   949,   528,  1747,    37,   118,\n",
      "        6871,   188,  7035,    17,   439,    14,    46,   274,   118,\n",
      "        6973,  8644,   329,    36,  1294,     3,    13,     6, 25200,\n",
      "       10963, 32093,  1294,   811,   302,   342, 17225,   147,    52,\n",
      "          30,  1786,    14,    34,   122,    52,   176, 23570,    25,\n",
      "         302,  1741, 12366,   122,   115, 32092,   329, 32091,   302,\n",
      "         190,  1855, 22210,   185, 27355,    14,  3639,    55, 15627,\n",
      "          47,    17,    97,    33,    46,  2427,   103,  1473,  3766,\n",
      "       32090, 12095,   494,   784,  1472,    14,     6,   194, 32089,\n",
      "        3535, 20074, 16303,  9271, 13261,   337,    14,  7169,  2027,\n",
      "        1848,  2293,  6996,  1294,  5018,    13, 14415, 26173,   494,\n",
      "        9655, 12498,    17,   329,    36,   322,     3,   904,    17,\n",
      "         118,  8772, 25917,  6619,    22,  7083,    42,  1908,  3370,\n",
      "        9730,    14,   291,  2293,  6996,  1294,   232,  5874, 32088,\n",
      "       19972,    65,   581,   113,  1741,  5696,   322,    22,  1755,\n",
      "           3,   517, 14510,   418,     7,  3884,   385,  2271,    13,\n",
      "        3608, 14598,  3241,   158,   262,  7599,    36,  3185,   906,\n",
      "          71, 32087,    13,     7,  3922, 11350, 32086,     6,   254,\n",
      "        1259,  2293,  6996,  1294,    34,  2695,  6900,   391, 16105,\n",
      "        1024,    14,   115,  7958,   718, 32085,    14,    25, 30589,\n",
      "        1024,    17, 12551,    14,   115,   315,   811,  2293,    34,\n",
      "          85,  2226,   591,    22,   232,   169,  1682,  3127,    25,\n",
      "         505,  7035,  1294, 15332,    14,     6,   194,     3,  5322,\n",
      "        3386,  3535, 20074,   221,    14,   596,  1750,    90,  2230,\n",
      "        2437,  2293,  6996,  1294,    24,  1494,  1688, 32084,    65,\n",
      "         108,  6860,   494,  2030,    36,  1763,  1384,     3, 20607,\n",
      "        2262,   266,  6996,  1294,  3176,   233,  2916, 21039,  6996,\n",
      "        1294,   138,    14,  9532,  8837, 32083,  1719,  5505,  3176,\n",
      "        2065,    71,   506,  4058,  1950,    14,  6943,  6074, 12572,\n",
      "          22,   506, 13174,    22,  3609,  4235, 10979,  4235,  1097,\n",
      "        4339,   428, 32082,   233,  1708,  2548,   156,  2916,    85,\n",
      "         100, 13641,  2421,   596,  1477,  9876, 32081,   739,   368,\n",
      "       32080,     7,  4731, 32079,    14, 32078,    42,   926,  1640,\n",
      "          37,  1245, 28369,   428,  6943,    22,  4235, 10979, 16007,\n",
      "        2390,    38,    25,   470,   596,    17, 32077,   233,  7582,\n",
      "           3,    13,     6,   254,  5505,  3176,  2065,  4235, 10979,\n",
      "          22,  6943, 12572,     3,  4596,    34,  3006,  1652,   553,\n",
      "       32076,    67,  3358,   108,  1602,   109,   408,   281,    34,\n",
      "           3,   202,    14,   721,    67,   582,   665,    34,    71,\n",
      "        2242, 32075,    22,    28,  1602,  2242,  4058,  1950,    14,\n",
      "          46, 32074,   136,   233,   156,  2916,    14,     6, 32073,\n",
      "        1427, 32072,    89,    34,     3, 27904,    14, 13641,  3768,\n",
      "          45,  3247, 21560,  3270,   596,    22,  5880,     7,   377,\n",
      "        3450,   116,    25,   190,    17,   736,    71,   138,     3,\n",
      "       15053,    14,  6943, 12572,  2681,  1524,  4058,  1950, 16007,\n",
      "        2390,    71,  5880, 32071,   233,  3768,     3,    13,     6,\n",
      "         254, 16956,  2065, 32070,   182, 25079, 32069,     1]), 'targets': array([32099,  1608, 13762,   302,  1615,    36,  1763, 17662,    22,\n",
      "        6996,  1294,    17,   150,   470,   302, 32098,   797, 32097,\n",
      "         869, 13841,    36,  2175,  6996,  1294, 32096,   418,     7,\n",
      "        3884, 32095, 13216,    47, 32094,   113, 32093,    56,  2293,\n",
      "        6996, 32092,    30,   118, 18766, 27767,    17, 32091,   105,\n",
      "         257,     3,  5144, 32090,    28,   330, 32089,     3,  1822,\n",
      "          37,    14,  4648, 17662,    14, 32088,   811, 32087,   302,\n",
      "           3, 32086, 16010,    13, 32085,    65,   221,     3,    13,\n",
      "           6, 19850, 32084,   158,   310, 32083,  2233,  6224,   644,\n",
      "       32082,   470,   820, 32081,   266, 32080,   349,  5443, 32079,\n",
      "           3, 21039,  6996,  1294,   547,  4754, 32078,    17,  2214,\n",
      "       32077,   427,  4458, 11607,    24, 32076,   470,    67,   528,\n",
      "       32075, 10006,    22,  2293, 32074,   470,    67,   348,   751,\n",
      "       32073,   194,    71, 10480,    24, 32072,  2856, 32071,  6996,\n",
      "        1294,   212,    25,   751, 32070,    71, 32069, 21072, 32068,\n",
      "         271, 32067,   157,    24,   292,   338,    13, 32066, 17299,\n",
      "          65,   108, 32065,    55,   233, 32064,  3111,    31, 32063,\n",
      "        1294,  1427, 32062, 19074,     5, 32061,    55, 32060,   257,\n",
      "       32059, 14822,  6889, 32058,    25,  2437,     3,    13,     6,\n",
      "        2598,  1604,  2544, 32057,   651,  2138,    21, 18049,    22,\n",
      "        6586,    24, 32056,    36, 32055,  1011,    89, 32054,   558,\n",
      "          30,  5263, 20030, 32053,   313,   221, 32052,  4677,  3323,\n",
      "         145,   842, 32051, 22045, 32050,    30, 32049,   296, 32048,\n",
      "          87,   334,     7, 32047,   700,    24, 32046,    13,    16,\n",
      "        4402,  3896, 32045, 28152,    17, 31234, 32044,    22, 32043,\n",
      "          30,   758,  6676,     1])}\n",
      "{'inputs': array([  232, 32099,    13,     4, 18830,  1881,    55, 11034,     5,\n",
      "          73, 10356, 30680,  2112,   266,   869, 32098,   147,    17,\n",
      "         116,     3, 10066,    38,    14,    34, 10948,  5637,  1592,\n",
      "          60,   432,  1643,    17,  7010,   532,    30, 13939,  1488,\n",
      "          17,   807,   147,    22,  4511, 32097,     3, 32096,   417,\n",
      "        3502,  1905, 32095, 10178,   261, 18370,    13,     4, 32094,\n",
      "         306,     5, 11286,     3, 13576,  1206, 10178,   261, 18370,\n",
      "           7, 24688, 10866, 18146, 17167,    47,    13,     4, 10302,\n",
      "         306,     7, 15696,     5, 11286, 32093,   133,   471,  5446,\n",
      "       14180,    31,  6761,    30,  8533,  1159,  1994, 25118,   138,\n",
      "       21356,  1723, 18192,   746, 23575, 14787, 19269,  1402,  2946,\n",
      "          24,   462,  1295,  2362,  8168,   909,    13,     6,   417,\n",
      "        3502,  1905,  1206, 10178,   261, 13464,    13,     4, 10302,\n",
      "         306,   380,     5,  2129,  4417,     3, 13576,  1206, 10178,\n",
      "         261, 14208, 32092,   417,  7430,     5,  2129,  4417, 32091,\n",
      "       19786,   221,    24,  7683,    16,   100,   344,  7360,  6173,\n",
      "          24, 32090,  3497,  1125,  4550, 17050,    13,     4, 11044,\n",
      "          75,     5, 32089,    36,   100,  1011,    89,    34, 32088,\n",
      "        3528,   221,    14,   240,   744,    17,  2592,    93,   358,\n",
      "        4867,  5163,     7, 23649,    50,  2652,   452,  4014,  3494,\n",
      "          17,   391,   342,  4893,  3420,  2158, 15286, 15255,    13,\n",
      "           4, 29966, 32087,    24,   145,    14,   494,  6133,  4867,\n",
      "        4014,  3494,    17,  3420,  2158, 12274, 30674,    17, 14075,\n",
      "          14,   494,   919,    47,  3494,    17,   807, 22665,    24,\n",
      "         884, 20868,  3494,    24,   521,   138,     3,    13,     6,\n",
      "        4409,  1510, 20501, 30680,    13,     4,  4409,  8168,   909,\n",
      "           5,    93,  2471,    28,   779,  1469,   683,  2845, 18664,\n",
      "        7539, 14769,    47,  1133,    13,     4,   347, 10444,     5,\n",
      "       12274, 32086,  1604,  6836,   432,  6761,     3,    13,     6,\n",
      "       27996,  6173,   116, 32085,    12, 18840,  5257,  4253,  2673,\n",
      "         159,  1506,   396, 13694,    13,     4,   201,     5,   180,\n",
      "        1172,   575,     3,   548,    53,  1803,   515, 30680,    14,\n",
      "          24,   185,    13,    12, 12058, 32084, 14502,   103, 32083,\n",
      "         140,  5916,   438, 13735,    87, 22743,    91, 11301,    25,\n",
      "        3175,  8036,    12,     3,    13,     6, 31671,    14,  6761,\n",
      "          62, 20038,  4253, 11762,   159,   212,    28,   440,   125,\n",
      "          13,    12, 27150, 32082,   454,    53,  1363,   515,  1600,\n",
      "        1830,    21,  2669, 10046, 32081,  5322,  3386,  1097,  2771,\n",
      "       32080,   153,    14,    13,  1909,  8345,   765, 12274,   280,\n",
      "        1812, 14078,    22, 27190,   445, 18779,  1338, 30905,    22,\n",
      "       21286, 23211,    24,  1870,  2125, 23250, 16624,     3,    13,\n",
      "           6, 27312,    34,   807,  7068,    24,  1263,  9395,    38,\n",
      "       32079,   665,    13,    12,  1172,  9381,  2597,    12,    22,\n",
      "          13,    12, 32078,  8168,   414,    63,    12,   268,  3867,\n",
      "        3404,    71,  2036,   105,   598,    22, 15673,    24,  4440,\n",
      "         103,    14,     6,   194,     3,    13, 22322,    24,   604,\n",
      "        1487,   398,   274,   221,    13,    12,  1744,   549,    12,\n",
      "        8432, 14787,    60,  5571,  1402,    24,   139,  2067,   351,\n",
      "         462,  2225,     3, 29821,   313,  6676,  4451,    36,  1979,\n",
      "        8247,   491,   348,  3867,    46,   438, 32077,   200,  8066,\n",
      "         358,  6796,  1723,    17, 32076,  1065,    34,     3,   335,\n",
      "        1798, 14787,    17,   313, 32075,  1104,    73,  2112,   266,\n",
      "          22,  6627,   116,  3400,     3, 30680, 32074,  3339, 20718,\n",
      "          25,  8233,   334,   230, 12884, 21815,   351,    14, 12274,\n",
      "       30674,     3,  6761,    22,    13,  2839,  2544,  3025,  1930,\n",
      "          12, 18840, 11166,  2038,  1477,   507,   672,    28,  5543,\n",
      "        4796,     3,   246,   394,  1325,  4700,     3,     1]), 'targets': array([32099,   695,  6627, 32098,  1273,  2649,    14,  1305,   139,\n",
      "        6173, 32097,    36,  1979,   432,    17,  4033, 32096,    13,\n",
      "           6, 32095,  1206, 32094, 10302, 32093,     3,    13, 16427,\n",
      "       32092,  7729,    13,     4, 32091,     3, 32090,  8349, 32089,\n",
      "          14,     6,   194, 32088,     3, 32087,     5, 32086,     7,\n",
      "       28824,    17, 10158,  4245,     7, 32085,  2137,   765,   874,\n",
      "        6761,     7,  2986, 32084,  3002,    12,  8810,    13,    12,\n",
      "         829, 32083,    22, 32082,  4640,    13,     4,  3848,     5,\n",
      "         207, 32081, 18105,   138,    12,    14,     6,   194,     3,\n",
      "       32080,    81, 32079,   854,  3818,    17,  1666,  5423,  2043,\n",
      "           7,  6889,   103, 32078,   199, 32077,  5716,    71,   608,\n",
      "           3, 13525, 32076,  3365,  2802,   746,  1436,    46,   348,\n",
      "       32075,    33,   424,  1045,  1019,   186,  2836, 32074,  1363,\n",
      "        2836,  6761, 32073, 12274,    22, 32072,   180,   675,     3,\n",
      "       10164, 32071,   158,   368, 11130, 11286,    13, 32070, 10706,\n",
      "       32069,   820, 32068,  8317,    24,  1839,    14, 32067,  3022,\n",
      "       32066, 28932, 32065,  4685, 10164, 32064,    34,    22, 32063,\n",
      "         820, 32062,    45, 32061,   375,  3225, 32060,     6, 32059,\n",
      "        1481,  6663, 32058,  3549, 10005, 13711, 32057,  5684, 32056,\n",
      "        1076,    93, 32055,  1347,   226,  1105,  1273,  2327,     3,\n",
      "           1])}\n"
     ]
    }
   ],
   "source": [
    "nq_task = t5.data.TaskRegistry.get('dumping_dataset')\n",
    "ds = nq_task.get_dataset(\n",
    "    split = 'train', sequence_length = {'inputs': 512, 'targets': 512}\n",
    ")\n",
    "for ex in tfds.as_numpy(ds.take(5)):\n",
    "    print(ex)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def question_dataset(split, shuffle_files = False):\n",
    "    del shuffle_files\n",
    "    ds = tf.data.TextLineDataset(\n",
    "        [\n",
    "            'gs://mesolitica-tpu-general/t5-data-v2/qa.tsv',\n",
    "        ]\n",
    "    )\n",
    "\n",
    "    ds = ds.map(\n",
    "        functools.partial(\n",
    "            tf.io.decode_csv,\n",
    "            record_defaults = ['', ''],\n",
    "            field_delim = '\\t',\n",
    "            use_quote_delim = False,\n",
    "        ),\n",
    "        num_parallel_calls = tf.data.experimental.AUTOTUNE,\n",
    "    )\n",
    "    ds = ds.map(lambda *ex: dict(zip(['question', 'answer'], ex)))\n",
    "    return ds\n",
    "\n",
    "\n",
    "def question_preprocessor(ds):\n",
    "    def to_inputs_and_targets(ex):\n",
    "        return {\n",
    "            'inputs': tf.strings.join(['soalan: ', ex['question']]),\n",
    "            'targets': ex['answer'],\n",
    "        }\n",
    "\n",
    "    return ds.map(\n",
    "        to_inputs_and_targets,\n",
    "        num_parallel_calls = tf.data.experimental.AUTOTUNE,\n",
    "    )\n",
    "\n",
    "\n",
    "t5.data.TaskRegistry.remove('question_dataset')\n",
    "t5.data.TaskRegistry.add(\n",
    "    'question_dataset',\n",
    "    dataset_fn = question_dataset,\n",
    "    splits = ['train'],\n",
    "    text_preprocessor = [question_preprocessor],\n",
    "    sentencepiece_model_path = vocab,\n",
    "    postprocess_fn = t5.data.postprocessors.lower_text,\n",
    "    metric_fns = [t5.evaluation.metrics.accuracy],\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'inputs_plaintext': b'soalan: yang memenangi liga juara sebanyak 2020 pada tahun 2013?', 'inputs': array([2762,   31,   17, 1477, 4983, 5805,  739, 3229,   33,   53,  646,\n",
      "         77,    1]), 'targets_plaintext': b'Orang India Mumbai', 'targets': array([ 1033,   688, 17801,     1])}\n",
      "{'inputs_plaintext': b'soalan: di mana jaren jackson senior bermain bola keranjang kolej?', 'inputs': array([ 2762,    31,    24,   185,    13,  3410,   258,    13, 15469,\n",
      "         729,  2000,   964,  1078,  7951,  4509,    77,     1]), 'targets_plaintext': b'Universiti Georgetown', 'targets': array([ 1908, 17322,     1])}\n",
      "{'inputs_plaintext': b'soalan: siapa yang menyanyikan anak lelaki yang baik?', 'inputs': array([ 2762,    31,  1652,    17, 10835,   270,   257,    17,   187,\n",
      "          77,     1]), 'targets_plaintext': b'Cockerel Chorus', 'targets': array([ 858, 5389,  607, 4782, 5901,    1])}\n",
      "{'inputs_plaintext': b'soalan: siapa penyanyi asal saya menembak sheriff?', 'inputs': array([ 2762,    31,  1652,  4002,  3203,    67,  3956, 14008,    77,\n",
      "           1]), 'targets_plaintext': b'The Wailers', 'targets': array([   35, 13731,  1530,    16,     1])}\n",
      "{'inputs_plaintext': b'soalan: siapa yang menyanyikannya untuk had untuk helang?', 'inputs': array([ 2762,    31,  1652,    17, 10835,    38,    25,   102,    25,\n",
      "          57,  2607,    77,     1]), 'targets_plaintext': b'Randy Meisner', 'targets': array([14938,  1019,    16,  1179,     1])}\n"
     ]
    }
   ],
   "source": [
    "nq_task = t5.data.TaskRegistry.get('question_dataset')\n",
    "ds = nq_task.get_dataset(\n",
    "    split = 'train', sequence_length = {'inputs': 256, 'targets': 32}\n",
    ")\n",
    "for ex in tfds.as_numpy(ds.take(5)):\n",
    "    print(ex)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pair_dataset(split, shuffle_files = False):\n",
    "    del shuffle_files\n",
    "    ds = tf.data.TextLineDataset(tf.io.gfile.glob('gs://mesolitica-tpu-general/t5-data-v2/*pair.tsv'))\n",
    "\n",
    "    ds = ds.map(\n",
    "        functools.partial(\n",
    "            tf.io.decode_csv,\n",
    "            record_defaults = ['', ''],\n",
    "            field_delim = '\\t',\n",
    "            use_quote_delim = False,\n",
    "        ),\n",
    "        num_parallel_calls = tf.data.experimental.AUTOTUNE,\n",
    "    )\n",
    "    ds = ds.map(lambda *ex: dict(zip(['text'], ex)))\n",
    "    return ds\n",
    "\n",
    "\n",
    "t5.data.TaskRegistry.remove('pair_dataset')\n",
    "t5.data.TaskRegistry.add(\n",
    "    'pair_dataset',\n",
    "    dataset_fn = pair_dataset,\n",
    "    splits = ['train'],\n",
    "    text_preprocessor = [prep.next_sentence_prediction],\n",
    "    sentencepiece_model_path = vocab,\n",
    "    postprocess_fn = t5.data.postprocessors.lower_text,\n",
    "    metric_fns = [t5.evaluation.metrics.accuracy],\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Entity <function neighboring_pairs.<locals>.<lambda> at 0x7ff0bc8d4048> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: Failed to parse source code of <function neighboring_pairs.<locals>.<lambda> at 0x7ff0bc8d4048>, which Python reported as:\n",
      "      lambda x: x[text_key], num_parallel_calls=tf.data.experimental.AUTOTUNE)\n",
      "\n",
      "If this is a lambda function, the error may be avoided by creating the lambda in a standalone statement.\n",
      "WARNING: Entity <function neighboring_pairs.<locals>.<lambda> at 0x7ff0bc8d4048> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: Failed to parse source code of <function neighboring_pairs.<locals>.<lambda> at 0x7ff0bc8d4048>, which Python reported as:\n",
      "      lambda x: x[text_key], num_parallel_calls=tf.data.experimental.AUTOTUNE)\n",
      "\n",
      "If this is a lambda function, the error may be avoided by creating the lambda in a standalone statement.\n",
      "{'inputs_plaintext': b'nsp: Bahasa yang ditekankan, dipetik dengan baik di Serbin. menyiratkan bahawa pihak yang tidak berada dalam persaingan langsung.', 'inputs': array([   13,   152,  4615,    31,  4730,    17, 21621,   103,    14,\n",
      "       23968,    28,   187,    24, 25847,   153,     3, 27406,    56,\n",
      "         432,    17,    30,   288,    36,  5563,  1320,     3,     1]), 'targets_plaintext': b'next', 'targets': array([554,   1])}\n",
      "{'inputs_plaintext': b'nsp: Walaupun ragu-ragu mengenai kecenderungan *C. auris* untuk menyebabkan jangkitan saluran kencing atau empyema seperti yang dinyatakan oleh CDC, kami mendapati bahawa UTI adalah tapak kedua paling biasa bagi jangkitan simptomatik \\\\[[@CR20]\\\\].', 'inputs': array([   13,   152,  4615,    31,   629, 11074,     7,  9282,   137,\n",
      "       12519,   863,   241,     3,    21,  4672,    16,  2608,    25,\n",
      "         864, 10943,  3354, 18394,    87, 16814,  3070,   527,   105,\n",
      "          17,  4277,    60, 13126,    14,   154,  1608,    56,   180,\n",
      "        4691,    52,  6932,   334,   279,   503,   158, 10943, 30938,\n",
      "          83,  2095,    13,     2, 28269,  1407,   241,   586,  1016,\n",
      "         332,     2,   332,     3,     1]), 'targets_plaintext': b'next', 'targets': array([554,   1])}\n",
      "{'inputs_plaintext': b'nsp: Model-model yang berbeza dipersembahkan dalam [Sec. Saya tidak menggunakan Linux, tetapi saya yakin anda mungkin juga boleh mendapatkan pemandu.', 'inputs': array([   13,   152,  4615,    31,  5614,     7, 13983,    17,   936,\n",
      "       22487,    36,   356,    75,  4791,     3,   217,    30,   311,\n",
      "       23994,    14,   113,    67,  2319,    70,   167,    93,   150,\n",
      "         751,  2720,     3,     1]), 'targets_plaintext': b'not_next', 'targets': array([   69,  2902, 18304,    41,     1])}\n",
      "{'inputs_plaintext': b'nsp: dan Cohen et al. Granola _tropika dengan mangga kering dan halia_.', 'inputs': array([   13,   152,  4615,    31,    22,  7812,    13,   666,  1781,\n",
      "           3, 13328,  5176,    13,  2902, 12564,  7080,    28,    13,\n",
      "         527,  7345,  6935,    22,  1553,   925,  2902,     3,     1]), 'targets_plaintext': b'not_next', 'targets': array([   69,  2902, 18304,    41,     1])}\n",
      "{'inputs_plaintext': b'nsp: Adakah anda mempunyai soalan lain sebelum anda pergi \"? Dia mencuba sekali lagi untuk mencarinya di muka.', 'inputs': array([  13,  152, 4615,   31, 1222,   70,  118, 2762,  116,  281,   70,\n",
      "        657,   13,    6,   77,  160, 3990,  793,  221,   25,  627,   38,\n",
      "         24, 6249,    3,    1]), 'targets_plaintext': b'next', 'targets': array([554,   1])}\n"
     ]
    }
   ],
   "source": [
    "nq_task = t5.data.TaskRegistry.get('pair_dataset')\n",
    "ds = nq_task.get_dataset(\n",
    "    split = 'train', sequence_length = {'inputs': 256, 'targets': 32}\n",
    ")\n",
    "for ex in tfds.as_numpy(ds.take(5)):\n",
    "    print(ex)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def news_dataset(split, shuffle_files = False):\n",
    "    del shuffle_files\n",
    "    ds = tf.data.TextLineDataset(\n",
    "        [\n",
    "            'gs://mesolitica-tpu-general/t5-data-v2/newstitle.tsv'\n",
    "        ]\n",
    "    )\n",
    "\n",
    "    ds = ds.map(\n",
    "        functools.partial(\n",
    "            tf.io.decode_csv,\n",
    "            record_defaults = ['', ''],\n",
    "            field_delim = '\\t',\n",
    "            use_quote_delim = False,\n",
    "        ),\n",
    "        num_parallel_calls = tf.data.experimental.AUTOTUNE,\n",
    "    )\n",
    "    ds = ds.map(lambda *ex: dict(zip(['question', 'answer'], ex)))\n",
    "    return ds\n",
    "\n",
    "\n",
    "def news_preprocessor(ds):\n",
    "    def to_inputs_and_targets(ex):\n",
    "        return {\n",
    "            'inputs': tf.strings.join(['tajuk: ', ex['question']]),\n",
    "            'targets': ex['answer'],\n",
    "        }\n",
    "\n",
    "    return ds.map(\n",
    "        to_inputs_and_targets,\n",
    "        num_parallel_calls = tf.data.experimental.AUTOTUNE,\n",
    "    )\n",
    "\n",
    "\n",
    "t5.data.TaskRegistry.remove('news_dataset')\n",
    "t5.data.TaskRegistry.add(\n",
    "    'news_dataset',\n",
    "    dataset_fn = news_dataset,\n",
    "    splits = ['train'],\n",
    "    text_preprocessor = [news_preprocessor],\n",
    "    sentencepiece_model_path = vocab,\n",
    "    postprocess_fn = t5.data.postprocessors.lower_text,\n",
    "    metric_fns = [t5.evaluation.metrics.accuracy],\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'inputs_plaintext': b'tajuk: Mahkamah Tinggi di sini hari ini menetapkan 1 Mac bagi pengurusan kes petisyen pilihan raya bagi kerusi Parlimen Kimanis dan Sipitang yang dimenangi calon Barisan Nasional (BN) pada Pilihan Raya Umum Ke-14 (PRU-14), 9 Mei lalu. Timbalan Pendaftar Mahkamah Tinggi Kota Kinabalu, Cindy Mc Juce Balitus menetapkan tarikh itu bagi Parlimen Kimanis yang akan didengar di hadapan Hakim Datuk Lee Heng Cheong di Mahkamah Tinggi di sini. Dalam pendengaran pengurusan kes dalam kamar itu, Suruhanjaya Pilihan Raya (SPR) diwakili peguam Faizal Sarbi dan Abdul Fikri Jaafar. Calon Parti Warisan Sabah (WARISAN) bagi Parlimen Kimanis, Datuk Karim Bujang diwakili Syaiful Sufeyyan Sidin dan Ahli Parlimen Kimanis, Datuk Seri Anifah Aman diwakili Wilson Chang Khai Sim serta Rizwandean M Borhan. Kelmarin, Mahkamah Persekutuan di Putrajaya memerintahkan perbicaraan petisyen pilihan raya diadakan bagi kerusi Parlimen Kimanis dan Sipitang yang dimenangi calon BN pada PRU-14. Panel lima hakim diketuai, Ketua Hakim Sabah dan Sarawak, Datuk Seri David Wong Dak Wah, berkata merit pilihan raya Kimanis perlu didengar memandangkan calon WARISAN di Parlimen Kimanis, mematuhi peraturan petisyen pilihan raya. Sementara di mahkamah berasingan, Pesuruhjaya Kehakiman, Ismail Ibrahim, menetapkan 1 Mac ini bagi sebutan kes petisyen pilihan raya Parlimen Sipitang yang akan didengar di hadapan Hakim Azhahari Kamal Ramli di Mahkamah Tinggi. SPR diwakili Faizal manakala Ahli Parlimen Sipitang, Yamani Hafez Musa, diwakili Rizwandean dan Azhar Farhan Arisin manakala peguam mewakili calon WARISAN, Noor Hayaty Mustapha, tidak hadir. Pada PRU-14, Anifah yang juga bekas Menteri Luar memenangi kerusi Parlimen Kimanis dengan majoriti tipis 156 undi, iaitu 11,942 undi berbanding 11,786 undi diraih Karim. Bagi Parlimen Sipitang, Yamani yang juga anak bekas Ketua Menteri, Tan Sri Musa Aman menang kerusi itu dengan 852 undi majoriti, sekali gus menewaskan Noor Hayaty.', 'inputs': array([ 5749,    31,  1870,  4209,    24,   442,    89,    34,  3635,\n",
      "         179,   686,   158,  2357,   744, 16504,   507,   672,   158,\n",
      "        3160,  2174,  1889,    47,   346,    22,  2295,  5237,   750,\n",
      "          17, 18301,   699,  4931,  1133,    13,     4,  4157,     5,\n",
      "          33,  3243,  2686,  6596,   386,     7,  1313,    13,     4,\n",
      "       15078,     7,  1313,     5,    14,   674,  1019,   186,     3,\n",
      "        3415,  1794, 15301,  1870,  4209,  2345, 12657,    14, 20538,\n",
      "        2522,  3727,   841,  8241,  6203,  3635,  3123,    37,   158,\n",
      "        2174,  1889,    47,   346,    17,    45,  9559,    24,  1992,\n",
      "        3955,   390,  1898,   135,  2460,  2807,  1360,    24,  1870,\n",
      "        4209,    24,   442,     3,   335, 14185,  2357,   744,    36,\n",
      "       22832,    37,    14,  3549,  3243,  2686,    13,     4,    75,\n",
      "        5447,     5, 13742,  2449, 16241,  5865,  1405,    22,  1097,\n",
      "          13,  2397,  8536, 17579,     3,  4804,  1779, 13107,  1467,\n",
      "          13,     4, 16813,  5382,  3272,     5,   158,  2174,  1889,\n",
      "          47,   346,    14,   390, 15099,  2503,  6205, 13742,  9532,\n",
      "          91,  1989,  1327,  2388,   128,  4976, 16823,   153,    22,\n",
      "        2171,  2174,  1889,    47,   346,    14,   390,   738,   923,\n",
      "       15317,  5034, 13742,  4297, 10709, 20876,  5789,   494, 21850,\n",
      "        2020,   429,    47,   499,  6650,  1936,     3,  7646,  1957,\n",
      "         153,    14,  1870,  1950,    24,  5949,  7925,  3872, 16504,\n",
      "         507,   672,  2770,   158,  3160,  2174,  1889,    47,   346,\n",
      "          22,  2295,  5237,   750,    17, 18301,   699,  3568,    33,\n",
      "       11026,     7,  1313,     3, 16902,   741,  3165,  7920,    14,\n",
      "         803,  3955,  1467,    22,  1839,    14,   390,   738,   927,\n",
      "        7791, 20838, 16555,    14,   291, 14397,   507,   672,  1889,\n",
      "          47,   346,   315,  9559,  5166,   699, 12212, 10218,  3272,\n",
      "          24,  2174,  1889,    47,   346,    14,  6870,  1592, 16504,\n",
      "         507,   672,     3,  1822,    24,  1624,  7756,    14, 11219,\n",
      "        7743,    14,  2759,  2884,    14,  3635,   179,   686,    34,\n",
      "         158, 18120,   744, 16504,   507,   672,  2174,  2295,  5237,\n",
      "         750,    17,    45,  9559,    24,  1992,  3955,  4343,  1256,\n",
      "        2663, 15661, 13346,    24,  1870,  4209,     3,   235,  5447,\n",
      "       13742, 16241,  3435,  2171,  2174,  2295,  5237,   750,    14,\n",
      "       10354,    91, 22903,  4585, 10202,    14, 13742, 21850,  2020,\n",
      "         429,    47,    22, 14318, 29062,  9526,  3777,  3435,  2449,\n",
      "        2860,   699, 12212, 10218,  3272,    14,  6402, 27503,   128,\n",
      "       28499,    14,    30,  3028,     3,   206, 11026,     7,  1313,\n",
      "          14,   923, 15317,    17,    93,   821,   506,  4727,  1477,\n",
      "        3160,  2174,  1889,    47,   346,    28,  5003, 14201,   454,\n",
      "         396,  3494,    14,   931,  6584,  5286,   215,  3494,  1325,\n",
      "        6584,   433,  5829,  3494,    24,  5345,   261, 15099,     3,\n",
      "        1616,  2174,  2295,  5237,   750,    14, 10354,    91,    17,\n",
      "          93,   270,   821,   803,   506,    14,  1544,  1526, 10202,\n",
      "        5034,  2338,  3160,    37,    28,  8791,   215,  3494,  5003,\n",
      "          14,   793,  7919, 12884,  6402, 27503,   128,     3,     1]), 'targets_plaintext': b'1 Mac pengurusan kes petisyen pilihan raya Parlimen Kimanis, Sipitang', 'targets': array([  179,   686,  2357,   744, 16504,   507,   672,  2174,  1889,\n",
      "          47,   346,    14,  2295,  5237,   750,     1])}\n"
     ]
    }
   ],
   "source": [
    "nq_task = t5.data.TaskRegistry.get('news_dataset')\n",
    "ds = nq_task.get_dataset(\n",
    "    split = 'train', sequence_length = {'inputs': 1024, 'targets': 1024}\n",
    ")\n",
    "for ex in tfds.as_numpy(ds.take(1)):\n",
    "    print(ex)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def summarization_dataset(split, shuffle_files = False):\n",
    "    del shuffle_files\n",
    "    ds = tf.data.TextLineDataset(\n",
    "        [\n",
    "            'gs://mesolitica-tpu-general/t5-data-v2/summarization.tsv'\n",
    "        ]\n",
    "    )\n",
    "\n",
    "    ds = ds.map(\n",
    "        functools.partial(\n",
    "            tf.io.decode_csv,\n",
    "            record_defaults = ['', ''],\n",
    "            field_delim = '\\t',\n",
    "            use_quote_delim = False,\n",
    "        ),\n",
    "        num_parallel_calls = tf.data.experimental.AUTOTUNE,\n",
    "    )\n",
    "    ds = ds.map(lambda *ex: dict(zip(['question', 'answer'], ex)))\n",
    "    return ds\n",
    "\n",
    "\n",
    "def summarization_preprocessor(ds):\n",
    "    def to_inputs_and_targets(ex):\n",
    "        return {\n",
    "            'inputs': tf.strings.join(['ringkasan: ', ex['question']]),\n",
    "            'targets': ex['answer'],\n",
    "        }\n",
    "\n",
    "    return ds.map(\n",
    "        to_inputs_and_targets,\n",
    "        num_parallel_calls = tf.data.experimental.AUTOTUNE,\n",
    "    )\n",
    "\n",
    "\n",
    "t5.data.TaskRegistry.remove('summarization_dataset')\n",
    "t5.data.TaskRegistry.add(\n",
    "    'summarization_dataset',\n",
    "    dataset_fn = summarization_dataset,\n",
    "    splits = ['train'],\n",
    "    text_preprocessor = [summarization_preprocessor],\n",
    "    sentencepiece_model_path = vocab,\n",
    "    postprocess_fn = t5.data.postprocessors.lower_text,\n",
    "    metric_fns = [t5.evaluation.metrics.accuracy],\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'inputs_plaintext': b\"ringkasan: Seorang wanita Texas dijatuhi hukuman penjara seumur hidup tanpa pembebasan bersyarat Isnin kerana mencekik anak perempuan tunangnya hingga mati setelah dia menemui teks-teks perkauman di telefonnya dan mengakhiri hubungan itu. Juri mendapati Melinda Muniz, kini berusia 26 tahun, membunuh Grace Ford pada 9 Januari 2014 - beberapa jam selepas bapa Grace, seorang veteran Perang Iraq, menyuruhnya berpindah keluar dari pangsapuri Plano, Texas, mereka. 'Anda adalah jahat tulen,' nenek Grace berkata kepada Muniz dalam satu kenyataan impak di mahkamah, WFAA melaporkan. 'Anda tidak akan dapat menyentuh anak lagi. 'Perbicaraan pembunuhan: Melinda Muniz, 26, (kiri) didakwa membunuh anak perempuan tunangnya, Grace (kanan) Grace, dua tahun, yang juga mengalami kecederaan parah pada alat kelaminnya, meninggal dunia ketika dia dikeluarkan sokongan hidup tiga hari kemudian di hospital. Muniz tidak memberi keterangan pada perbicaraan, di mana juri mendengar dia telah mengubah ceritanya tentang apa yang berlaku pada malam itu beberapa kali. Peguam beliau telah berehat kes tersebut tanpa mengemukakan bukti atau memanggil mana-mana saksi lain, lapor KDFW. 'Tiada siapa yang menang di sini,' kata datuk Grace, Doug Reeves. 'Kami dapat hidup tanpa Rahmat. 'Tunangan Muniz Mitch Ford memberitahunya semasa pernyataan impaknya bahawa dia memaafkan dia, berkata 'Saya masih marah kepada anda. Tetapi, lebih daripada itu, saya sedih. ' Semasa perbicaraan, pendakwa raya berkata Muniz pada asalnya memberitahu polis bahawa dia telah dirogol dan diketuk tidak sedarkan diri oleh penceroboh yang merapatkan mulut Grace. Bagaimanapun, selepas ujian menyangkal ceritanya, Muniz mendakwa dia menghidap amnesia dan tidak dapat mengingati dua tahun lalu, lapor Star Local Media. Horrific: Grace was found face-down on the bed, lifeless, with duct tape over her mouth. Dalam temu bual ketiga, pendakwa raya berkata, Muniz berkata Grace meletakkan pita saluran di atas mulutnya sehingga Muniz panik dan mengetuk dirinya tidak sedarkan diri dengan kuali. Dia kemudian berkata bahawa dia menyedari Grace mempunyai pita di atas mulutnya, menarik ia, melihat dia tidak sedarkan diri, meletakkan kembali, dan mengetuk dirinya keluar. Katanya, dia 'tidak tahu' jika ia memba kemalangan atau sengaja, mahkamah mendengar. Siasatan sejak itu mendedahkan Muniz didakwa dilihat membeli pita saluran dan wayar zip beberapa jam sebelum Grace ditemui tidak bernyawa di atas katilnya dengan pita di atas mulut dan wayarnya di sekitar bilik, walaupun Muniz mendakwa ini adalah bahan seni dan kraf. Pencarian untuk 'duct tape' dan 'kill' juga didapati pada MacBook yang pulih dari harta yang dikongsi Muniz dengan Grace dan Ford, juri mendengar. Muniz, yang bertemu Ford pada September 2012, setahun setengah selepas Grace dilahirkan, telah bertindak sebagai penjaga rumahnya sejak bapanya memenangi hak penjagaan penuh pada 2013. Pasangan ini bertemu pada September 2012, setahun setengah selepas Grace dilahirkan. Babysat: Grace, yang digambarkan bersama ibunya Emily Ward, adalah babysat oleh Muniz apabila dia meninggal dunia. Beliau mendapat hak penjagaan penuh daripada bekas isterinya Emily Reeves Ward, yang menjadi ketagih alkohol dan pil preskripsi selepas beliau kembali dari perang Iraq keduanya dan hubungan mereka terputus. Ford berkata beliau mencadangkan kepada Muniz pada Disember 2013 tetapi menamatkan hubungan pada 9 Januari, pagi selepas beliau melihat beliau telah menghantar gambar-gambar beliau dalam pakaian dalam kepada pelatih peribadinya. Dia memberitahu mahkamah: 'Semasa saya bersiap untuk bekerja, saya memberitahunya bahawa saya menemui barang-barang di telefonnya,' dia memberi keterangan. ' Itu bukan hujah besar atau apa-apa. Saya kata ia tidak akan berjaya dan menghentikan pertunangan. Dia agak tidak emosional, seperti itu sebenarnya bukan masalah besar. 'Tidak ada pergaduhan. Saya menyediakan segala-galanya untuknya. Saya memberitahunya bahawa dia mempunyai seminggu [untuk berpindah]. Dia boleh menyimpan cincin pertunangan dan menjualnya dan menyimpan kereta untuk memandu dan mencari pekerjaan. Dia tidak menangis, tidak kelihatan marah. ' Perbicaraan pembunuhan yang dibuka pada Selasa, juga telah mendengar bukti daripada pegawai kecemasan pertama yang sampai ke tempat kejadian jenayah - yang memberi keterangan bahawa Muniz kelihatan berpura-pura tidak sedarkan diri. 'Dia mengambil nafas besar dan jenis perlahan-lahan jatuh ke lantai seolah-olah dia telah pengsan,' Pegawai Camille Bowie berkata. 'Matanya jenis fluttering, seperti dia cuba untuk memastikan mereka ditutup dengan paksa. Tiada pergerakan di mata dan badan apabila seseorang pengsan -- jika anda benar-benar pengsan, anda tidak akan meletakkan diri anda dengan lembut. Saya rasa ia tidak asli. ' Ford memberitahu mahkamah bahawa dia menghubungi polis selepas panggilan telefonnya dengan Muniz dipotong pendek. Katanya, dia berada di tempat kerja, Muniz menelefon untuk mengatakan dia baru pulang dengan Grace, dan telefon seolah-olah jatuh ke lantai. Beliau tiba di apartmen pada masa yang sama dengan pegawai. Muniz berbaring di bahagian depannya dengan seluar dalamnya di sekeliling buku lali. Grace berada di bilik tidur di hadapannya dan kekal tidak bertindak balas kerana dia dibawa ke ER di The Medical Center of Plano. Cerita: Muniz memberikan tiga akaun berbeza tentang apa yang berlaku sebelum ini dengan mengatakan bahawa dia tidak tahu sama ada kematian Grace adalah kemalangan atau sengaja. Dia diberi dos epinefrin yang memulakan jantungnya, kemudian doktor menyedari lebam dan darah di sekitar kawasan farajnya. 'Saya percaya Grace telah diserang secara seksual,' Dr Richard Honaker memberi keterangan. Deborah Davis, pengamal kesihatan yang memeriksa Muniz untuk bukti serangan seksual, berkata: 'Selain daripada kata kerja, saya melihat apa-apa untuk menunjukkan sebarang serangan seksual. Defendan tidak kelihatan beremosi, cukup tenang dan sangat bekerjasama. ' Muniz berkata, dia membeli pita saluran, gunting, tali zip dan sapu kapas dari Dollar Tree untuk projek seni dan kraf. Ketika dia mula membersihkan, dia berkata Grace mesti menjadi takut, berfikir Muniz hampir untuk vakum, jadi dia berlari ke bilik tidurnya dan pita mulutnya. Muniz menjelaskan bahawa dia memakai sarung tangan dan sebahagiannya mengeluarkan pita saluran dari mulut Grace untuk melihat sama ada dia OK. Apabila ia jelas Grace tidak bernafas, dia panik dan meletakkan pita kembali. Kemudian, ketika Grace tidak sedarkan diri, dia mengumpulkan gunting, tali zip dan pita saluran dan membuangnya di Dumpster melintasi tempat letak kereta, katanya. Dalam temu ramah yang dirakam dengan Muniz, dimainkan kepada juri, Detektif Chris Jones berkata: 'Saya akan terus jujur dengan anda: Terdapat isu-isu dengan cerita anda. Anda dibesarkan bahawa Grace mempunyai pita saluran padanya ketika itu tidak pernah disebutkan. Bagaimana anda tahu jika anda tidak sedarkan diri? ' Tambahnya, tempoh masa itu tidak realistik, mengatakan tempoh enam minit antara panggilan 911 dan ketibaan pihak berkuasa 'tidak realistik' bagi penceroboh melakukan serangan fizikal dan seksual terhadap dua orang.\", 'inputs': array([7680,   47,   31, ...,  177,   17,    1]), 'targets_plaintext': b'Melinda Muniz didapati bersalah membunuh Grace Ford yang berusia dua tahun. Gadis itu ditemui di sebuah apartmen Plano, Texas, mati dengan pita saluran yang menutup mulut dan luka pada alat kelaminnya pada Januari 2014. Muniz yang kini berusia 26 tahun ditemui di lantai dengan seluar dalam di sekeliling buku lali. Dia mendakwa seorang penceroboh menyerang mereka berdua dan mengetuknya. Polis berkata dia kelihatan seperti berpura-pura tidak sedarkan diri. Muniz kemudian mengaku tidak diserang, memberikan dua cerita lain. Muniz pernah menjaga Grace sementara tunangnya Mitch Ford sedang bekerja. Ford membuang Muniz pagi itu selepas menemui teks-teks yang tidak senonoh di telefonnya. Muniz telah ditetapkan untuk mengambil pendirian untuk memberi keterangan tetapi menolak. Dia dijatuhi hukuman penjara seumur hidup tanpa pembebasan bersyarat.', 'targets': array([29348,  8126,  4262,  4847,  3465,  1862, 10585,  2727,    17,\n",
      "         547,   192,    53,     3, 17051,    37,  2822,    24,   136,\n",
      "       14533,  4934,   162,    14,  1389,    14,  1035,    28, 12120,\n",
      "        3354,    17,  3444,  6916,    22,  9490,    33,  1829, 27937,\n",
      "          38,    33,  1247,   444,     3,  8126,  4262,    17,   427,\n",
      "         547,  1447,    53,  2822,    24,  6000,    28,  8424,    36,\n",
      "          24, 20163,   978,  2683,   567,     3,   160,  2285,   163,\n",
      "        1627, 16633,  2659,    46,  8776,    22, 13602,    38,     3,\n",
      "        1481,   291,    61,  1043,   105, 17564,     7,  7780,    30,\n",
      "        5994,   103,   322,     3,  8126,  4262,   365,  3857,    30,\n",
      "        8251,    14,   438,   192,  2535,   116,     3,  8126,  4262,\n",
      "         317,  2576, 10585,   746, 29432, 14232,  2727,   579,   616,\n",
      "           3,  2727,  3550,  8126,  4262,   971,    37,   428,  2332,\n",
      "        5409,     7,  9127,    16,    17,    30, 22518,    24,  1034,\n",
      "          38,     3,  8126,  4262,    62,  4629,    25,   330,  9786,\n",
      "          25,   262,  4869,   113,  1469,     3,   160, 23467,  2088,\n",
      "        2133,  8547,   591,   401,  9313, 25802,     3,     1])}\n"
     ]
    }
   ],
   "source": [
    "nq_task = t5.data.TaskRegistry.get('summarization_dataset')\n",
    "ds = nq_task.get_dataset(\n",
    "    split = 'train', sequence_length = {'inputs': 1024, 'targets': 1024}\n",
    ")\n",
    "for ex in tfds.as_numpy(ds.take(1)):\n",
    "    print(ex)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def similarity_dataset(split, shuffle_files = False):\n",
    "    del shuffle_files\n",
    "    ds = tf.data.TextLineDataset(\n",
    "        [\n",
    "            'gs://mesolitica-tpu-general/t5-data-v2/snli.tsv',\n",
    "            'gs://mesolitica-tpu-general/t5-data-v2/mnli.tsv'\n",
    "        ]\n",
    "    )\n",
    "\n",
    "    ds = ds.map(\n",
    "        functools.partial(\n",
    "            tf.io.decode_csv,\n",
    "            record_defaults = ['', ''],\n",
    "            field_delim = '\\t',\n",
    "            use_quote_delim = False,\n",
    "        ),\n",
    "        num_parallel_calls = tf.data.experimental.AUTOTUNE,\n",
    "    )\n",
    "    ds = ds.map(lambda *ex: dict(zip(['question', 'answer'], ex)))\n",
    "    return ds\n",
    "\n",
    "\n",
    "def similarity_preprocessor(ds):\n",
    "    def to_inputs_and_targets(ex):\n",
    "        return {\n",
    "            'inputs': ex['question'],\n",
    "            'targets': ex['answer'],\n",
    "        }\n",
    "\n",
    "    return ds.map(\n",
    "        to_inputs_and_targets,\n",
    "        num_parallel_calls = tf.data.experimental.AUTOTUNE,\n",
    "    )\n",
    "\n",
    "\n",
    "t5.data.TaskRegistry.remove('similarity_dataset')\n",
    "t5.data.TaskRegistry.add(\n",
    "    'similarity_dataset',\n",
    "    dataset_fn = similarity_dataset,\n",
    "    splits = ['train'],\n",
    "    text_preprocessor = [similarity_preprocessor],\n",
    "    sentencepiece_model_path = vocab,\n",
    "    postprocess_fn = t5.data.postprocessors.lower_text,\n",
    "    metric_fns = [t5.evaluation.metrics.accuracy],\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'inputs_plaintext': b'ayat1: Seorang lelaki sedang duduk di kerusi hijau, bercakap di telefon, dan bekerja di komputer riba, dan terdapat tapak pembinaan bersebelahan dengan bangunannya. ayat2: Lelaki itu boleh mendengar pembinaan yang sedang berjalan.', 'inputs': array([13694,   201,    31,  1064,   257,   579,  1625,    24,  3160,\n",
      "        4359,    14,   962,    24,  1034,    14,    22,   616,    24,\n",
      "        2411, 15021,    14,    22,   697,  6932,  2825, 17609,    28,\n",
      "        1527,    38,     3, 13694,   215,    31,  5561,    37,   150,\n",
      "        1605,  2825,    17,   579,  1047,     3,     1]), 'targets_plaintext': b'neutral', 'targets': array([12712,     1])}\n"
     ]
    }
   ],
   "source": [
    "nq_task = t5.data.TaskRegistry.get('similarity_dataset')\n",
    "ds = nq_task.get_dataset(\n",
    "    split = 'train', sequence_length = {'inputs': 256, 'targets': 32}\n",
    ")\n",
    "for ex in tfds.as_numpy(ds.take(1)):\n",
    "    print(ex)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def en_ms_dataset(split, shuffle_files = False):\n",
    "    del shuffle_files\n",
    "    ds = tf.data.TextLineDataset(\n",
    "        [\n",
    "            'gs://mesolitica-tpu-general/t5-data-v2/en-ms.tsv'\n",
    "        ]\n",
    "    )\n",
    "\n",
    "    ds = ds.map(\n",
    "        functools.partial(\n",
    "            tf.io.decode_csv,\n",
    "            record_defaults = ['', ''],\n",
    "            field_delim = '\\t',\n",
    "            use_quote_delim = False,\n",
    "        ),\n",
    "        num_parallel_calls = tf.data.experimental.AUTOTUNE,\n",
    "    )\n",
    "    ds = ds.map(lambda *ex: dict(zip(['question', 'answer'], ex)))\n",
    "    return ds\n",
    "\n",
    "\n",
    "def en_ms_preprocessor(ds):\n",
    "    def to_inputs_and_targets(ex):\n",
    "        return {\n",
    "            'inputs': tf.strings.join(['terjemah Inggeris ke Melayu: ', ex['question']]),\n",
    "            'targets': ex['answer'],\n",
    "        }\n",
    "\n",
    "    return ds.map(\n",
    "        to_inputs_and_targets,\n",
    "        num_parallel_calls = tf.data.experimental.AUTOTUNE,\n",
    "    )\n",
    "\n",
    "\n",
    "t5.data.TaskRegistry.remove('en_ms_dataset')\n",
    "t5.data.TaskRegistry.add(\n",
    "    'en_ms_dataset',\n",
    "    dataset_fn = en_ms_dataset,\n",
    "    splits = ['train'],\n",
    "    text_preprocessor = [en_ms_preprocessor],\n",
    "    sentencepiece_model_path = vocab,\n",
    "    postprocess_fn = t5.data.postprocessors.lower_text,\n",
    "    metric_fns = [t5.evaluation.metrics.accuracy],\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'inputs_plaintext': b\"terjemah Inggeris ke Melayu: Right, it's not even an email.\", 'inputs': array([   13, 26087,  2040,    55,  1550,    31,  8471,    14,    43,\n",
      "          12,    16,    69,   318,    80,  4083,     3,     1]), 'targets_plaintext': b'Betul, itu bukan nya e-mail.', 'targets': array([17232,    14,    37,   232,    13,    38,    13,    81,     7,\n",
      "        4114,     3,     1])}\n"
     ]
    }
   ],
   "source": [
    "nq_task = t5.data.TaskRegistry.get('en_ms_dataset')\n",
    "ds = nq_task.get_dataset(\n",
    "    split = 'train', sequence_length = {'inputs': 1024, 'targets': 1024}\n",
    ")\n",
    "for ex in tfds.as_numpy(ds.take(1)):\n",
    "    print(ex)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def ms_en_dataset(split, shuffle_files = False):\n",
    "    del shuffle_files\n",
    "    ds = tf.data.TextLineDataset(\n",
    "        [\n",
    "            'gs://mesolitica-tpu-general/t5-data-v2/ms-en.tsv'\n",
    "        ]\n",
    "    )\n",
    "\n",
    "    ds = ds.map(\n",
    "        functools.partial(\n",
    "            tf.io.decode_csv,\n",
    "            record_defaults = ['', ''],\n",
    "            field_delim = '\\t',\n",
    "            use_quote_delim = False,\n",
    "        ),\n",
    "        num_parallel_calls = tf.data.experimental.AUTOTUNE,\n",
    "    )\n",
    "    ds = ds.map(lambda *ex: dict(zip(['question', 'answer'], ex)))\n",
    "    return ds\n",
    "\n",
    "\n",
    "def ms_en_preprocessor(ds):\n",
    "    def to_inputs_and_targets(ex):\n",
    "        return {\n",
    "            'inputs': tf.strings.join(['terjemah Melayu ke Inggeris: ', ex['question']]),\n",
    "            'targets': ex['answer'],\n",
    "        }\n",
    "\n",
    "    return ds.map(\n",
    "        to_inputs_and_targets,\n",
    "        num_parallel_calls = tf.data.experimental.AUTOTUNE,\n",
    "    )\n",
    "\n",
    "\n",
    "t5.data.TaskRegistry.remove('ms_en_dataset')\n",
    "t5.data.TaskRegistry.add(\n",
    "    'ms_en_dataset',\n",
    "    dataset_fn = ms_en_dataset,\n",
    "    splits = ['train'],\n",
    "    text_preprocessor = [ms_en_preprocessor],\n",
    "    sentencepiece_model_path = vocab,\n",
    "    postprocess_fn = t5.data.postprocessors.lower_text,\n",
    "    metric_fns = [t5.evaluation.metrics.accuracy],\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'inputs_plaintext': b'terjemah Melayu ke Inggeris: Meliputi pelbagai genre, filem-filemnya termasuk \"Near Dark\" (1987), \"Point Break\" (1991), \"Strange Days\" (1995), \"K-19: The Widowmaker\" (2002), \"The Hurt Locker\" (2008), \"Zero Dark Thirty\" (2012), dan \"Detroit\" (2017).', 'inputs': array([   13, 26087,  1550,    55,  2040,    31,   777,  5610, 13612,\n",
      "         879,  7664,    14,   492,     7,  9733,    38,   293,    13,\n",
      "           6, 13934,   382, 11786,     6,    13,     4,   756,  4406,\n",
      "           5,    14,    13,     6, 21341,    13, 17035,     6,    13,\n",
      "           4,   756,  5569,     5,    14,    13,     6,  8138, 10484,\n",
      "        1714,    16,     6,    13,     4,   756,  2920,     5,    14,\n",
      "          13,     6,   471,     7,   756,    31,    35,  3075, 13524,\n",
      "        9965,     6,    13,     4, 23882,     5,    14,    13,     6,\n",
      "         198,    13, 16025,    41,  1743,  5389,     6,    13,     4,\n",
      "       14089,     5,    14,    13,     6,  1757,  5418, 11786, 26881,\n",
      "           6,    13,     4, 15780,     5,    14,    22,    13,     6,\n",
      "        3469,  3125,   545,     6,    13,     4, 14931,     5,     3,\n",
      "           1]), 'targets_plaintext': b'Covering a variety of genres, his films include \"Near Dark\" (1987), \"Point Break\" (1991), \"Strange Days\" (1995), \"K-19: The Widowmaker\" (2002), \"The Hurt Locker\" ( 2008), \"Zero Dark Thirty\" (2012), and \"Detroit\" (2017).', 'targets': array([20056,    63,    21,  5547,    18,  7664,    16,    14,    68,\n",
      "        5195,  1531,    13,     6, 13934,   382, 11786,     6,    13,\n",
      "           4,   756,  4406,     5,    14,    13,     6, 21341,    13,\n",
      "       17035,     6,    13,     4,   756,  5569,     5,    14,    13,\n",
      "           6,  8138, 10484,  1714,    16,     6,    13,     4,   756,\n",
      "        2920,     5,    14,    13,     6,   471,     7,   756,    31,\n",
      "          35,  3075, 13524,  9965,     6,    13,     4, 23882,     5,\n",
      "          14,    13,     6,   198,    13, 16025,    41,  1743,  5389,\n",
      "           6,    13,     4,  1046,     5,    14,    13,     6,  1757,\n",
      "        5418, 11786, 26881,     6,    13,     4, 15780,     5,    14,\n",
      "          20,    13,     6,  3469,  3125,   545,     6,    13,     4,\n",
      "       14931,     5,     3,     1])}\n"
     ]
    }
   ],
   "source": [
    "nq_task = t5.data.TaskRegistry.get('ms_en_dataset')\n",
    "ds = nq_task.get_dataset(\n",
    "    split = 'train', sequence_length = {'inputs': 1024, 'targets': 1024}\n",
    ")\n",
    "for ex in tfds.as_numpy(ds.take(1)):\n",
    "    print(ex)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "def knowledge_graph_dataset(split, shuffle_files = False):\n",
    "    del shuffle_files\n",
    "    ds = tf.data.TextLineDataset(\n",
    "        [\n",
    "            'gs://mesolitica-tpu-general/t5-data-v2/knowledge-graph.tsv'\n",
    "        ]\n",
    "    )\n",
    "\n",
    "    ds = ds.map(\n",
    "        functools.partial(\n",
    "            tf.io.decode_csv,\n",
    "            record_defaults = ['', ''],\n",
    "            field_delim = '\\t',\n",
    "            use_quote_delim = False,\n",
    "        ),\n",
    "        num_parallel_calls = tf.data.experimental.AUTOTUNE,\n",
    "    )\n",
    "    ds = ds.map(lambda *ex: dict(zip(['question', 'answer'], ex)))\n",
    "    return ds\n",
    "\n",
    "def knowledge_graph_preprocessor(ds):\n",
    "    def to_inputs_and_targets(ex):\n",
    "        return {\n",
    "            'inputs': tf.strings.join(['grafik pengetahuan: ', ex['question']]),\n",
    "            'targets': ex['answer'],\n",
    "        }\n",
    "\n",
    "    return ds.map(\n",
    "        to_inputs_and_targets,\n",
    "        num_parallel_calls = tf.data.experimental.AUTOTUNE,\n",
    "    )\n",
    "\n",
    "t5.data.TaskRegistry.remove('knowledge_graph_dataset')\n",
    "t5.data.TaskRegistry.add(\n",
    "    'knowledge_graph_dataset',\n",
    "    dataset_fn = knowledge_graph_dataset,\n",
    "    splits = ['train'],\n",
    "    text_preprocessor = [knowledge_graph_preprocessor],\n",
    "    sentencepiece_model_path = vocab,\n",
    "    postprocess_fn = t5.data.postprocessors.lower_text,\n",
    "    metric_fns = [t5.evaluation.metrics.accuracy],\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'inputs_plaintext': b'grafik pengetahuan: Connie Dierking bermain untuk Sacramento Kings dan Philadelphia Tapers dalam Liga Bola Keranjang Kebangsaan. Dia mewakili Amerika Syarikat.', 'inputs': array([12333,  5836,    31,  1597,  5217,   208,   125,  2921,   964,\n",
      "          25, 19188, 13423,    22,  4093, 17082,   287,    36,  4529,\n",
      "        7910, 31548,  1697,     3,   160,  2860,   209,   487,     3,\n",
      "           1]), 'targets_plaintext': b\"Connie Dierking country for sport United States, member of sports team Philadelphia Tapers, competition class men's basketball, member of sports team Sacramento Kings.\", 'targets': array([ 1597,  5217,   208,   125,  2921,   286,    29,  6177,   485,\n",
      "         891,    14,  1988,    18,  3364,   650,  4093, 17082,   287,\n",
      "          14,  3081,  2816,   440,    12,    16,  9272,    14,  1988,\n",
      "          18,  3364,   650, 19188, 13423,     3,     1])}\n"
     ]
    }
   ],
   "source": [
    "nq_task = t5.data.TaskRegistry.get('knowledge_graph_dataset')\n",
    "ds = nq_task.get_dataset(\n",
    "    split = 'train', sequence_length = {'inputs': 1024, 'targets': 1024}\n",
    ")\n",
    "for ex in tfds.as_numpy(ds.take(1)):\n",
    "    print(ex)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "def paraphrase_dataset(split, shuffle_files = False):\n",
    "    del shuffle_files\n",
    "    ds = tf.data.TextLineDataset(\n",
    "        [\n",
    "            'gs://mesolitica-tpu-general/t5-data-v2/paraphrase.tsv'\n",
    "        ]\n",
    "    )\n",
    "\n",
    "    ds = ds.map(\n",
    "        functools.partial(\n",
    "            tf.io.decode_csv,\n",
    "            record_defaults = ['', ''],\n",
    "            field_delim = '\\t',\n",
    "            use_quote_delim = False,\n",
    "        ),\n",
    "        num_parallel_calls = tf.data.experimental.AUTOTUNE,\n",
    "    )\n",
    "    ds = ds.map(lambda *ex: dict(zip(['question', 'answer'], ex)))\n",
    "    return ds\n",
    "\n",
    "def paraphrase_preprocessor(ds):\n",
    "    def to_inputs_and_targets(ex):\n",
    "        return {\n",
    "            'inputs': tf.strings.join(['parafrasa: ', ex['question']]),\n",
    "            'targets': ex['answer'],\n",
    "        }\n",
    "\n",
    "    return ds.map(\n",
    "        to_inputs_and_targets,\n",
    "        num_parallel_calls = tf.data.experimental.AUTOTUNE,\n",
    "    )\n",
    "\n",
    "t5.data.TaskRegistry.remove('paraphrase_dataset')\n",
    "t5.data.TaskRegistry.add(\n",
    "    'paraphrase_dataset',\n",
    "    dataset_fn = paraphrase_dataset,\n",
    "    splits = ['train'],\n",
    "    text_preprocessor = [paraphrase_preprocessor],\n",
    "    sentencepiece_model_path = vocab,\n",
    "    postprocess_fn = t5.data.postprocessors.lower_text,\n",
    "    metric_fns = [t5.evaluation.metrics.accuracy],\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'inputs_plaintext': b'parafrasa: Pada bulan November, Royals memperoleh CF Coco Crisp dari Boston Red Sox sebagai pertukaran untuk RP Ramon Ramirez.', 'inputs': array([  445,  4435,   722,    31,   206,   204,   664,    14,  2586,\n",
      "          16,  1243,    13, 13547, 19576, 16469,  4615,    42,  1010,\n",
      "        1975, 11434,    85,  4137,    25,    13, 15240, 24817, 23171,\n",
      "           3,     1]), 'targets_plaintext': b'Pada bulan November, Royals CF Coco Crisp diperoleh dari Boston Red Sox sebagai pertukaran untuk RP Ramon Ramirez.', 'targets': array([  206,   204,   664,    14,  2586,    16,    13, 13547, 19576,\n",
      "       16469,  4615,  5578,    42,  1010,  1975, 11434,    85,  4137,\n",
      "          25,    13, 15240, 24817, 23171,     3,     1])}\n"
     ]
    }
   ],
   "source": [
    "nq_task = t5.data.TaskRegistry.get('paraphrase_dataset')\n",
    "ds = nq_task.get_dataset(\n",
    "    split = 'train', sequence_length = {'inputs': 1024, 'targets': 1024}\n",
    ")\n",
    "for ex in tfds.as_numpy(ds.take(1)):\n",
    "    print(ex)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
